import time

import requests
from lxml import etree

headers = {
    "accept-language": "zh-CN,zh;q=0.9",
    "referer": "https://www.reuters.com/business/",
    "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, Gecko) Chrome/134.0.0.0 Safari/537.36"
}


def get_section_id(url):
    response = requests.get(url, headers=headers)

    # 对响应进行解码
    html_str = response.content.decode()

    # xpath初始化
    root = etree.HTML(html_str)
    # 提取特定节点
    item_list = root.xpath("//ul[@class='section-selector-links__list__1R6dg']")

    links = []  # 创建一个空列表来存储链接
    for item in item_list:
        # 提取链接
        link_items = item.xpath("./li/a/@href")
        links.extend(link_items)  # 将提取到的链接添加到列表中
    return links  # 返回所有的链接


def get_index_url(url):
    response = requests.get(url, headers=headers)

    # 对响应进行解码
    html_str = response.content.decode()

    # xpath初始化
    root = etree.HTML(html_str)
    # 提取特定节点
    item_list = root.xpath("//ul[@class='nav-bar__nav-list__1eoTx']")
    link_list = []  # 创建一个空列表来存储链接
    for item in item_list:
        # 提取链接
        links = item.xpath(
            "./li[@class='nav-bar__nav-item__1p3l9 nav-bar__category__16MHp']/div[@class='nav-bar__nav-button__2LWkE']/a/@href")
        link_list.extend(links)  # 将提取到的链接添加到列表中
    return link_list  # 返回所有的链接


if __name__ == '__main__':
    base_url = "https://www.reuters.com"
    link_list = get_index_url(base_url)
    all_section_ids = []  # 创建一个空列表来存储所有 section_id 的链接
    for link in link_list:
        url = base_url + link

        section_ids = get_section_id(url)
        all_section_ids.extend(section_ids)  # 将每个 url 返回的 section_id 添加到列表中

    # 打印所有的 section_id 链接
    print(all_section_ids)  # 59个


    #自取即可了
    # links_list=['/world/africa/', '/world/americas/', '/world/asia-pacific/', '/world/china/', '/world/europe/', '/world/india/', '/world/israel-hamas/', '/world/japan/', '/world/middle-east/', '/world/ukraine-russia-war/', '/world/uk/', '/world/us/', '/world/reuters-next/', '/business/aerospace-defense/', '/business/autos-transportation/', '/business/davos/', '/business/energy/', '/business/environment/', '/business/finance/', '/business/healthcare-pharmaceuticals/', '/business/media-telecom/', '/business/retail-consumer/', '/business/future-of-health/', '/business/future-of-money/', '/business/take-five/', '/business/world-at-work/', '/markets/asia/', '/markets/carbon/', '/markets/commodities/', '/markets/currencies/', '/markets/deals/', '/markets/emerging/', '/markets/etf/', '/markets/europe/', '/markets/funds/', '/markets/global-market-data/', '/markets/rates-bonds/', '/markets/stocks/', '/markets/us/', '/markets/wealth/', '/markets/macromatters/', '/sustainability/boards-policy-regulation/', '/sustainability/climate-energy/', '/sustainability/land-use-biodiversity/', '/sustainability/society-equity/', '/sustainability/sustainable-finance-reporting/', '/sustainability/the-switch/', '/sustainability/reuters-impact/', '/sustainability/cop/', '/legal/government/', '/legal/legalindustry/', '/legal/litigation/', '/legal/transactional/', '/legal/us-supreme-court/', '/breakingviews/predictions/', '/technology/artificial-intelligence/', '/technology/cybersecurity/', '/technology/space/', '/technology/disrupted/']
